package com.push.spider.spider;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.HanLP;
import com.push.common.config.TransConfig;
import com.push.common.core.domain.AjaxResult;
import com.push.common.downloader.HttpClientDownloader;
import com.push.common.manager.BeanManager;
import com.push.common.utils.DateUtils;
import com.push.common.utils.EncryptUtil;
import com.push.common.utils.MD5;
import com.push.common.utils.StringUtils;
import com.push.common.utils.httpclient.HttpClientUtil;
import com.push.spider.utils.UrlUtils;
import com.push.web.base.BaseCrawler;
import com.push.web.base.BaseSpider;
import com.push.web.domain.BizArticle;
import com.push.web.domain.BizArticleAppendix;
import com.push.web.domain.BizLabelSecond;
import com.push.web.domain.BizSource;
import com.push.web.service.IBizArticleAppendixService;
import com.push.web.service.IBizArticleService;
import com.push.web.service.IBizLabelSecondService;
import com.push.web.service.IBizSourceService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.utils.HttpConstant;

import java.util.*;

import static com.push.common.utils.StringUtils.delHtmlTags;

public class CommSpider extends BaseCrawler {
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private String charset = "UTF-8";

    private BizSource source;

    private List<BizLabelSecond> labelSeconds = new ArrayList<>();

    @Autowired
    IBizArticleService articleService = BeanManager.getBean(IBizArticleService.class);

    @Autowired
    IBizSourceService sourceService = BeanManager.getBean(IBizSourceService.class);

    @Autowired
    IBizLabelSecondService labelSecondService = BeanManager.getBean(IBizLabelSecondService.class);

    @Autowired
    IBizArticleAppendixService articleAppendixService = BeanManager.getBean(IBizArticleAppendixService.class);

    @Autowired
    private TransConfig transConfig = BeanManager.getBean(TransConfig.class);

    public int size = 0;

    private Site site = Site.me().setRetryTimes(3).setSleepTime(3000).setTimeOut(10000);

    public CommSpider(String sourceId) {
        this.source = sourceService.selectBizSourceBySourceId(Long.valueOf(sourceId));
        this.labelSeconds = labelSecondService.selectBizLabelSecondListByParentIds(this.source.getLabelIds());
    }

    @Override
    public Spider createCrawler() {
        //爬虫入口URL
        Request request = new Request(source.getSpiderUrl());
        request.addHeader(HEADER_USER_AGENT_KEY, USER_AGENT_CHROME_OS);
        BaseSpider spider = new BaseSpider(this);
        spider.setDownloader(new HttpClientDownloader());
        spider.addRequest(request);
        return spider;
    }

    @Override
    public void process(Page page) {
        System.out.println(source.getPageXpath());
        if (page.getUrl().regex(source.getPageXpath()).match()) {
            System.out.println(page.getUrl().toString());
            //对于从来没有检索的数据源，或者10天内没有检索过的数据源，遍历全部页码，否则，只检索前2页
            if(StringUtils.isNotEmpty(source.getPageDivXpath())){
                page.addTargetRequests(page
                        .getHtml()
                        .xpath(source.getPageDivXpath())
                        .links().all());
            }
            // 如果需要爬取详情页，则继续把详情页的链接添加到爬虫队列中。
            // 如还有其他列表需要抓取，则继续添加
            Html html = page.getHtml();
            //获取列表
            if (!StringUtils.isEmpty(source.getListXpath())) {
                List<String> listXpaths = Arrays.asList(source.getListXpath().split(";"));
                for (String listXpath:listXpaths){
                    System.out.println(html.toString());
                    List<Selectable> tagNodes = html.xpath(listXpath).nodes();
                    for (Selectable contentNode : tagNodes) {
                        String titleOrg = extractInfo(source.getTitleXpath(), contentNode);
                        String url = UrlUtils.buildUrl(String.valueOf(page.getUrl()), extractInfo(source.getDetailsXpath(), contentNode));
                        String avator = UrlUtils.buildUrl(String.valueOf(page.getUrl()),extractInfo(source.getAvatorXpath(), contentNode));
                        String publishTime = extractInfo(source.getPublishTimeXpath(), contentNode);
                        page.addTargetRequest(extractInfo(source.getDetailsXpath(), contentNode));
                        if (StringUtils.isEmpty(url)){
                            continue;
                        }
                        String dataKey = EncryptUtil.encrypt(url, "md5");
                        BizArticle article = buildArticle(dataKey, titleOrg, url, avator, publishTime, source.getSourceId());
                        BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
                        if (StringUtils.isNull(bizArticle)){
                            if (strContains(article.getTitleOrg(), labelSeconds)){
                                //自动翻译标题
                                if (StringUtils.isContainsChinese(article.getTitleOrg())){
                                    article.setTitleCn(article.getTitleOrg());
                                } else {
                                    String transWord = getTransResult(article.getTitleOrg(), "auto", "zh");
                                    JSONObject titleObj = JSONObject.parseObject(transWord);
                                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                    if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                        article.setTitleCn(titleArray.getJSONObject(0).getString("dst"));
                                    }
                                }
                                //自动翻译摘要
                                if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                                    String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                                    JSONObject titleObj = JSONObject.parseObject(transWord);
                                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                    if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                        article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                                    }
                                }

                                articleService.insertBizArticle(article);
                            }
                        } else {
                            article.setId(bizArticle.getId());
                            articleService.updateBizArticle(article);
                        }
                    }
                }
            }
        } else {
            // 如果有业务需要抓取详情页，则继续在这儿抓取。
            String url = String.valueOf(page.getUrl());
            Html html = page.getHtml();
            String dataKey = EncryptUtil.encrypt(url, "md5");
            String summaryOrg = extractInfo(source.getSummaryXpath(), html);
            String contentsOrg = extractInfo(source.getContentsXpath(), html);
            String publisher = extractInfo(source.getPublisherXpath(), html);
            String publishTime = extractInfo(source.getPublishTimeXpath(), html);
            String avator = "";
            List<String> contentXpathList = Arrays.asList(source.getContentsXpath().split(";"));
            if(contentXpathList.size() > 0){
                for (String contentXpath: contentXpathList){
                    List<Selectable> contentNodes = html.xpath(contentXpath).nodes();
                    if(contentNodes.size() > 0){
                        /*
                         * 单独获取图片
                         */
                        List<Selectable> imgNodes = contentNodes.get(0).xpath("//img/@src").nodes();
                        if (imgNodes.size() > 0) {
                            avator = UrlUtils.buildUrl(url, imgNodes.get(0).toString());
                        }
                        break;
                    }
                }
            }
            BizArticle article = updateArticle(dataKey, summaryOrg, contentsOrg, publisher, publishTime, avator, source.getSourceId());
            BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
            if (StringUtils.isNull(bizArticle)){
                if (strContains(article.getTitleOrg(), labelSeconds)){
                    articleService.insertBizArticle(article);
                }
            } else {
                article.setId(bizArticle.getId());

                //自动翻译摘要
                if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                    String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                    JSONObject titleObj = JSONObject.parseObject(transWord);
                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                    if (titleArray.size() > 0){
                        article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                    }
                }/* else if (StringUtils.isNotEmpty(article.getContentsCn())){
                    List<String> sentenceList = HanLP.extractSummary(article.getContentsCn(),10);
                    if(sentenceList.size() > 0){
                        String abstact = sentenceList.toString();
                        article.setSummaryOrg(abstact.substring(1,abstact.lastIndexOf("]")));
                        String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (titleArray.size() > 0){
                            article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }
                }*/

                articleService.updateBizArticle(article);
            }

            if(StringUtils.isNotEmpty(source.getFilesXpath())){
                List<String> filesXpathList = Arrays.asList(source.getFilesXpath().split(";"));
                List<BizArticleAppendix> appendices = new ArrayList<>();
                for (String fileXpath: filesXpathList){
                    List<Selectable> fileNodes = html.xpath(fileXpath).nodes();
                    if(fileNodes.size() > 0){
                        for (Selectable fileNode:fileNodes){
                            BizArticleAppendix appendix = new BizArticleAppendix();
                            appendix.setAppendixUrl(fileNode.toString());
                            appendix.setArticleId(article.getId());
                        }
                    }
                }
                if (!appendices.isEmpty()){
                    articleAppendixService.batchSave(appendices);
                }
            }
        }
    }

    @Override
    public Site getSite() {
        Set<Integer> acceptStatus = new HashSet<Integer>();
        acceptStatus.add(HttpConstant.StatusCode.CODE_200);
        acceptStatus.add(301);
        acceptStatus.add(403);
        site.setAcceptStatCode(acceptStatus);
        return site;
    }

    public int getSize(){
        return size;
    }

    public String extractInfo(String xpathString, Selectable contentNode){
        if (StringUtils.isEmpty(xpathString)){
            return "";
        }
        List<String> xpathList = Arrays.asList(xpathString.split(";"));
        for (String xpath: xpathList){
            List<Selectable> nodes = contentNode.xpath(xpath).nodes();
            if(nodes.size() > 0){
                return nodes.get(0).toString();
            }
        }
        return "";
    }

    public BizArticle buildArticle(String dataKey, String titleOrg, String url,String avator, String publishTime, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setTitleOrg(titleOrg);
        article.setUrl(url);
        article.setAvator(avator);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTime),publishTime));
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public BizArticle updateArticle(String dataKey, String summaryOrg, String contentsOrg, String publisher, String publishTime, String avator, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setSummaryOrg(summaryOrg);
        contentsOrg.replaceAll("visibility: hidden;", "visibility: visible;");
        article.setContentsOrg(contentsOrg);
        article.setContentsCn(StringUtils.delHtmlTags(contentsOrg));
        article.setPublisher(publisher);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTime),publishTime));
        article.setAvator(avator);
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public String getTransResult(String query, String from, String to) {
        Map<String, String> params = buildParams(query, from, to);
        return HttpClientUtil.doGet(transConfig.getTransUrl(), params,false).getContent();
    }

    private Map<String, String> buildParams(String query, String from, String to) {
        Map<String, String> params = new HashMap<String, String>();
        params.put("q", query);
        params.put("from", from);
        params.put("to", to);
        params.put("appid", transConfig.getAppid());
        // 随机数
        String salt = String.valueOf(System.currentTimeMillis());
        params.put("salt", salt);
        // 签名
        String src = transConfig.getAppid() + query + salt + transConfig.getSecurityKey();
        params.put("sign", MD5.md5(src));
        return params;
    }

    public static boolean strContains(String str, List<BizLabelSecond> labels){
        if (labels == null){
            return true;
        }
        if (labels.isEmpty()){
            return true;
        }
        if(StringUtils.isEmpty(str)){
            return false;
        }
        long len = labels.parallelStream().filter(label -> str.toLowerCase(Locale.ROOT).contains(label.getLabelName().toLowerCase(Locale.ROOT))).count();
        if (len > 0){
            return true;
        }
        return false;
    }
}
